/**
 * Alipay.com Inc.
 * Copyright (c) 2004-2022 All Rights Reserved.
 */
package com.example.spider.provider.why10w.persistent;

import cn.hutool.core.io.FileUtil;
import com.example.spider.provider.why10w.dto.WhyTaskResult;
import org.shoulder.core.util.JsonUtils;

import java.io.File;
import java.nio.charset.StandardCharsets;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;

/**
 * @deprecated 历史数据处理
 * @author lym
 */
public class Why10ResultFormatter {

    public void doFormat(String domain) {
        List<File> files = FileUtil.loopFiles("F:\\spider\\" + domain + "\\json", file -> file.getName().endsWith(".json"));
        Map<String, WhyTaskResult> whyTaskResultMap = files.stream().parallel()
                .map(FileUtil::readUtf8String)
                .map(json -> JsonUtils.parseObject(json, WhyTaskResult.class))
                .collect(Collectors.toMap(WhyTaskResult::getUrl, r -> r, (k1, k2) -> k2));

        // 保存到文件
        String fileName = "F:/spider/" + domain + "/result.dict";
        File file = FileUtil.touch(fileName);
        FileUtil.writeString(JsonUtils.toJson(whyTaskResultMap), file, StandardCharsets.UTF_8);
    }

    public static void main(String[] args) {
        String domain = "swgwsm.bmcx.com";
        new Why10ResultFormatter().doFormat(domain);
    }
}